/*package com.payne;

import org.apache.http.HttpHost;
import org.apache.http.message.BasicNameValuePair;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.utils.HttpConstant;

import java.util.HashMap;
import java.util.Map;

//爬取showMore信息
class ShowMoreSpider implements PageProcessor
{
    private Site site;

    public  ShowMoreSpider(String proxyStr, String cookie)
    {
        String[] tmp = proxyStr.split(":");
        HttpHost proxy = new HttpHost(tmp[1].substring(2), Integer.parseInt(tmp[2]), tmp[0]);

        this.site = Site.me().setRetryTimes(3).setHttpProxy(proxy).addHeader("Cookie", cookie).setSleepTime(100).setTimeOut(10 * 1000).setCharset("UTF-8")
                .setUserAgent("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36");
    }

    @Override
    public void process(Page page)
    {
        System.out.println(page.getJson().jsonPath("body").regex("u2022 (.*?)<").toString());
    }

    @Override
    public Site getSite()
    {
        return site;
    }
}

public class VideoCommentSpider {
    public static String proxyString = "http://XXX.XX.XXX.XXX:XXXX";

    public static void main(String[] args)
    {
        //获取Video页面的Cookie、ShowMoreURL、client_url、session_token
        String url = "https://www.youtube.com/watch?v=Xo94zT93fAY";
        VideoSpider vs = new VideoSpider(url, proxyString);


        //获取comment
        PageProcessor spider = new ShowMoreSpider(proxyString, vs.getCookie());

        NameValuePair[] values = new NameValuePair[2];
        values[0] = new BasicNameValuePair("client_url", vs.getClient_url());
        values[1] = new BasicNameValuePair("session_token", vs.getSession_token());

        Map nameValuePair = new HashMap();
        nameValuePair.put("nameValuePair", values);

        Request request = new Request(vs.getShowMoreURL());
        request.setExtras(nameValuePair);
        request.setMethod(HttpConstant.Method.POST);

        Spider.create(spider).thread(5).addRequest(request).run();
    }
}*/
